# pylint: disable=no-member
# -*- encoding: utf-8 -*-
'''
@File    :   onespider.py
@Time    :   2020/04/10 09:23:10
@Author  :   Kellan Fan 
@Version :   1.0
@Contact :   kellanfan1989@gmail.com
@Desc    :   None
'''

# here put the import lib
import time
from bs4 import BeautifulSoup 
from misc.openurl import OpenUrl
from misc.pg_client import Mypostgres
from log.create_logger import create_logger

logger = create_logger()
base_url = 'http://wufazhuce.com/article/'

def send_pg(para, connect):
    '''将数据写入数据库'''
    sql = "insert into onesite(article_id,title,autor,content) values (%s, %s, %s, %s)"
    code = connect.execute(sql,para)
    if code:
        logger.info('insert [{}] ok'.format(para))
    else:
        print('insert [{}] error,message: [{}]'.format(para, code))
        logger.error('insert [{}] error,message: [{}]'.format(para, code))

fail_time = 0
pg_conn = Mypostgres()
try:
    cur_last_id = int(pg_conn.execute('select article_id from onesite order by id desc limit 1')[0][0])
except:
    cur_last_id = 0

while True:
    cur_last_id += 1
    data = {}
    ourl = OpenUrl(base_url + str(cur_last_id))
    code,doc = ourl.run()
    if code == 200:
        soup = BeautifulSoup(doc, 'lxml')
        data['article_id'] = str(cur_last_id)
        data["title"] = soup.find('h2',class_='articulo-titulo').text.strip()
        data["autor"] = soup.find('p',class_='articulo-autor').text.strip()
        data["content"] = soup.find('div',class_='articulo-contenido').text.strip()
        
        try:
            send_pg(list(data.values()),pg_conn)
            logger.info("insert [{}] successful".format(data["title"]))
        except Exception as e:
            logger.error("insert [{0}] failed: [{1}]".format(data["title"],e))
            continue
        time.sleep(1)
    else:
        fail_time += 1
        logger.error("Get [{}] failed".format(cur_last_id))
        if fail_time > 30:
            break
